from concurrent.futures import ThreadPoolExecutor
import copy
from v4.Page import Page
import logging

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(filename)s line:%(lineno)d [%(levelname)s] %(message)s',
                    datefmt='%a, %d %b %Y %H:%M:%S')

logger = logging.getLogger()


class Spider(object):
    def __init__(self,
                 spider_module_list,
                 thread_num,
                 request_items):
        self.spider_module_list = spider_module_list
        self.thread_num = thread_num
        self.request_items = request_items
        self.spider_modules_list = []

    def spider_thread_execute(self, request_items, spider_module_list):
        for request_item in request_items:
            page = Page()
            page.set_request(request_item)
            for spider_module in spider_module_list:
                page.set_page_processor_result(None)
                page.set_download_result(None)
                spider_module.execute(page)

    def run(self):
        logger.info("初始化爬虫……")
        for i in range(0, self.thread_num):
            self.spider_modules_list.append([copy.deepcopy(self.spider_module_list), []])
        i = 0
        logger.info("分配请求任务……")
        for request_item in self.request_items:
            self.spider_modules_list[i][1].append(request_item)
            i = (i + 1) % self.thread_num

        with ThreadPoolExecutor(max_workers=self.thread_num) as t:
            for spider_modules in self.spider_modules_list:
                logger.info("添加爬虫至线程池……")
                future = t.submit(self.spider_thread_execute, spider_modules[1], spider_modules[0])
                exception = future.exception()
                if exception is not None:
                    raise exception
